package com.g2room.driving.grap.article.qg;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.article.to.ArticlesTo;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrabNrxDataByProxy {
	String imgPath = "nrx";
	public String baseUrl = "http://lady.97jie.net/B/HunWaiYouQing/";
	private String soruce = "女人巷";
	private int count = 0;
	static List<String> titles = GenericArti.getTitles();
	
	public static void main(String[] args) throws Exception{
		GrabNrxDataByProxy gkd = new GrabNrxDataByProxy();
//		gkd.getNav();
		
		List<Integer> ids = GenericArti.getLatestIds(GenericArti.getPublishTimeStr(1), gkd.soruce);
		// 抓取图片
		if(null != ids && ids.size() > 0) {
			HandleImages.getContentImgs(ids, gkd.baseUrl, gkd.imgPath);
		}
    }
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		Map<String, String> pages = new HashMap<String, String>();
		pages.put("http://lady.97jie.net/B/HunWaiYouQing/69.html", "口述实录");
		pages.put("http://lady.97jie.net/B/QingGanGuShi/14.html", "情感故事");
		pages.put("http://lady.97jie.net/B/ZhenBianHua/15.html", "枕边话");
		pages.put("http://lady.97jie.net/B/PoXiGuanXi/07.html", "婆媳关系");
		pages.put("http://lady.97jie.net/B/HunYinWuYu/35.html", "婚姻物语");
		pages.put("http://lady.97jie.net/B/LianAiBaoDian/22.html", "恋爱宝典");

        for(String key : pages.keySet()) {
	        String pageTotal = key.substring(key.lastIndexOf("/") + 1, key.lastIndexOf("."));
	        for(int i = 1; i<Integer.valueOf(pageTotal); i++) {
	        	String url = i==1 ? key.substring(0, key.indexOf("/" + pageTotal)) + "/" : key.replaceAll(pageTotal, i+"");
	        	boolean rs = true;
	        	String tn = pages.get(key);
	        	rs = getPage(url, tn);
	        	if(!rs) break;
	        }
        }
        System.out.println("本次抓取数据：" + count);
		
        List<Integer> ids = GenericArti.getLatestIds(GenericArti.getPublishTimeStr(1), soruce);
		// 抓取图片
		if(null != ids && ids.size() > 0) {
			// 抓取缩略图
			HandleImages.getContentImgs(ids, baseUrl, imgPath);
		}
		// 最后输出失败的链接到数据库
		GenericArti.insertError2Db(soruce);
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public boolean getPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName + "====" + url);
		Document doc = GenericGrap.getDoc(url, "gb2312");
		Elements els = doc.getElementById("JaList").getElementsByTag("li");
		List<ArticlesTo> artis = new ArrayList<ArticlesTo>();
		for(Element el : els) {
			try {
				Element titleEl = el.getElementsByTag("a").get(0);
				String title = titleEl.text();
				if(titles.contains(title)) {
					System.out.println("存在，跳过：" + title);
					if(artis.size() == 0) {
						// 后续抓取的时候，就存在则直接返回
						return false;
					}
				}
				Elements thumbEls = el.getElementsByTag("img");
				Element thumbEl = null;
				ArticlesTo arti = new ArticlesTo();
				if(thumbEls != null && thumbEls.size() > 0) {
					thumbEl = el.getElementsByTag("img").get(0);
					arti.setThumb(thumbEl.attr("src"));
				}
				arti.setTitle(title);
				arti.setSource(soruce);
				arti.setSourceUrl(titleEl.attr("href"));
				arti.setTypeName(typeName);
				try {
					Element publishTimeEl = el.getElementsByAttributeValue("class", "submark").get(0);
					arti.setPublishTime(publishTimeEl.text());

					Element tagEl = el.getElementsByAttributeValue("class", "aList-label").get(0);
					if(null != tagEl && tagEl.text().trim().length() > 0)
					arti.setTag(tagEl.text().trim().replaceAll(" ", ","));
				} catch (Exception e) {
					e.printStackTrace();
				}
				arti.setState(8);
				artis.add(arti);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(ArticlesTo arti : artis) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
        return true;
	}

	public void getDetail(ArticlesTo arti) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get detail data!  " + arti.getSourceUrl());
		try {
			Document doc = GenericGrap.getDoc(arti.getSourceUrl(), "gb2312");
			// 获取分页信息
			Elements els = doc.getElementsByAttributeValue("class", "totalpage");
			if(null != els && els.size()>0) els =els.get(0).getElementsByTag("a");
//			doc.getElementById("BAIDU_SSP__wrapper_u2202022_0").remove();
//			doc.getElementsByAttributeValue("class","ty").remove();
			// 删除底部5个div标签
			String content = getContent(doc);
			if(null != els && els.size()>2) {
				// 获取页码数
//				els.remove(0);
				for(Element el : els) {
					if(el.text().equals("尾")) {
						String href = el.attr("href");
						// http://lady.97jie.net/B/HunWaiYouQing/20160509/6043-2.html
						String pageTotal = href.substring(href.lastIndexOf("-")+1, href.lastIndexOf("."));
						for(int i = 1; i<Integer.valueOf(pageTotal); i++) {
				        	String url = href.substring(0, href.lastIndexOf("-")+1) + (i+1) + ".html";
							Document pageDoc = GenericGrap.getDoc(url, "gb2312");
							content += "\n"+getContent(pageDoc);;
				        }
					}
				}
			}
			content = GenericArti.handleSpace(HtmlUtil.delAllTag(content));
			
			arti.setContent("<div class=\"article\">" + content + "</div>");
			GenericArti.insert2Db(arti);
//			GenericGrap.titles.add(arti.getTitle());
			count++;
        } catch (Exception e) {
        	GenericArti.errorUrls.put(arti.getSourceUrl(), arti.getTypeName());
        	e.printStackTrace();
        	return;
        }
	}
	
	public static String getContent(Document doc) {
		Elements els = doc.getElementsByAttributeValue("class", "artText").get(0).children();
		Elements pEls = new Elements();
		for(Element el : els) {
			if(el.outerHtml().length()>10 && el.tag().toString().equalsIgnoreCase("p")) {
				pEls.add(el);
			}
		}
//		for(int i=0; i<5; i++) {
//			divEls.remove(divEls.size()-i-1);
//		}
//		// 删除顶部6个div标签
//		for(int i=0; i<6; i++) {
//			divEls.remove(0);
//		}
		return pEls.outerHtml();
	}
}
